# -*- coding: utf-8 -*-
# =============================================================================
#     FileName: redis_expire_helper.py
#         Desc: 用于清理Redis服务器上指定前缀的KEY，通过循环SCAN方式降低清理过程对服务器的影响。
#       Author: GGA
#        Email:
#     HomePage:
#      Version: 1.0.1
#   LastChange: 2020-07-31
#      History:
# =============================================================================

import logging
import logging.handlers
import optparse
import queue
import random
import time

import redis

from utils.logger_helper import LoggerHelper

APP_NAME = "redis_expire_helper"
APP_VERSION = "1.0"
logger = LoggerHelper.get_logger(logger_name=APP_NAME)


class RedisExpireHelper(object):
    def __init__(self, redis_host, redis_port=6379, redis_password="", db_index=0):
        self.redis_host = redis_host
        self.redis_port = redis_port
        self.redis_password = redis_password
        self.db_index = db_index
        self.redis_pool = redis.ConnectionPool(
            host=self.redis_host,
            port=self.redis_port,
            db=self.db_index,
            password=self.redis_password,
            max_connections=10
        )
        self.task_queue = queue.Queue()

    def get_redis_connection(self):
        """
        从Redis Pool中获取连接，避免重复建立Redis连接
        :return:
        """
        return redis.Redis(connection_pool=self.redis_pool, decode_responses=True)

    def check_connection(self):
        redis_conn = self.get_redis_connection()
        try:
            redis_conn.ping()
            return True
        except Exception as _:
            return False

    def scan_keys(self, key_match, start_post=0, batch_scan_size=1000):
        """
        范围扫描匹配的键值
        :param key_match:   要匹配的KEY模式
        :param start_post:  扫描开始位置点
        :param batch_scan_size:  扫描的数量
        :return: stop_post,match_keys
            stop_post: 本次扫描的结束位置
            match_keys：本次扫描匹配的键值
        """
        redis_conn = self.get_redis_connection()
        stop_post, match_keys = redis_conn.scan(start_post, key_match, batch_scan_size)
        return stop_post, match_keys

    @classmethod
    def get_random_expire_seconds(cls, min_expire_seconds: int, max_random_seconds: int):
        """
        基于最小过期时间和最大随机延迟时间来获取随机过期秒数
        :param min_expire_seconds:
        :param max_random_seconds:
        :return:
        """
        if max_random_seconds <= 0:
            max_random_seconds = 1
        if min_expire_seconds <= 0:
            min_expire_seconds = 1
        return min_expire_seconds + random.randint(0, max_random_seconds)

    def expire_key(self, key_name, min_expire_seconds, max_random_seconds):
        """
        通过最小过期时间+随机过期时间设置KEY过期，避免大量KEY集中过期影响服务器性能
        :param key_name: 要设置过期的KEY
        :param min_expire_seconds: 最小过期时间
        :param max_random_seconds: 随机过期时间
        :return:
        """
        redis_conn = self.get_redis_connection()
        if max_random_seconds <= 0:
            max_random_seconds = 1
        expire_seconds = min_expire_seconds + random.randint(0, max_random_seconds)
        logger.info("set key {} expired at {} seconds later.".format(
            key_name,
            expire_seconds
        ))
        redis_conn.expire(key_name, expire_seconds)

    def get_ttl_seconds(self, key_name):
        """
        返回指定键的过期时长(秒)，-2表示已过期,-1表示永不过期，正数表示剩余过期秒数。
        :param key_name: 要查看的KEY
        :return: 指定KEY的过期时长(秒)
        """
        redis_conn = self.get_redis_connection()
        ttl_seconds = redis_conn.ttl(name=key_name)
        logger.debug("key：{}的过期时间为{}".format(key_name, ttl_seconds))
        return ttl_seconds

    def ie_unexpired_key(self, key_name):
        """
        判断指定KEY是否是永不过期
        :param key_name:
        :return:
        """
        ttl_seconds = self.get_ttl_seconds(key_name=key_name)
        if ttl_seconds == -1:
            return True
        else:
            return False

    def ie_expired_key(self, key_name):
        """
        判断指定KEY是否是永不过期
        :param key_name:
        :return:
        """
        ttl_seconds = self.get_ttl_seconds(key_name=key_name)
        if ttl_seconds == -2:
            return True
        else:
            return False

    def get_idle_seconds(self, key_name):
        """
        获取指定键的空闲时长。
        OBJECT IDLETIME <key> 返回给定 key 自储存以来的空转时间(idle， 没有被读取也没有被写入)，以秒为单位
        :param key_name:
        :return:
        """
        redis_conn = self.get_redis_connection()
        idle_seconds = redis_conn.object("IDLETIME", key=key_name)
        if idle_seconds is None:
            idle_seconds = 0
        logger.debug("key：{}的空闲时间为{}".format(key_name, idle_seconds))
        return idle_seconds

    def is_use_less_key(self, key_name, use_less_seconds):
        """
        判断指定键是否长期未使用
        :param key_name:  指定的键名
        :param use_less_seconds:  长期未使用的阈值(秒)
        :return:
        """
        idle_seconds = self.get_idle_seconds(key_name=key_name)
        if idle_seconds > use_less_seconds:
            return True
        else:
            return False

    def clean_unexpired_key(self, key_match, min_expire_seconds, max_random_seconds,
                            batch_scan_size=10000, batch_sleep_seconds=0.1):
        """
        批量清理指定模式的未设置过期键值。
        :param key_match: 键模式
        :param min_expire_seconds: 设置最小过期时长（秒）
        :param max_random_seconds: 设置最大随机时长(秒)
        :param batch_scan_size: 每次扫描行数
        :param batch_sleep_seconds: 每次扫描间歇（秒）
        :return:
        """
        start_post = 0
        logger.info("start to scan.")
        while True:
            stop_post, match_keys = self.scan_keys(
                key_match=key_match,
                start_post=start_post,
                batch_scan_size=batch_scan_size
            )
            logger.info("star post:{},stop post:{},scan keys:{},find keys:{}".format(
                start_post,
                stop_post,
                batch_scan_size,
                len(match_keys)
            ))
            for key_name in match_keys:
                if self.ie_unexpired_key(key_name=key_name):
                    self.expire_key(
                        key_name=key_name,
                        min_expire_seconds=min_expire_seconds,
                        max_random_seconds=max_random_seconds
                    )
            if stop_post == 0:
                break
            start_post = stop_post
            time.sleep(batch_sleep_seconds)
        logger.info("stop to scan.")

    def clean_expired_key(self, key_match, batch_scan_size=1000, batch_sleep_seconds=0.1):
        """
        批量清理指定模式的已过期键值。
        :param key_match: 键模式
        :param batch_scan_size: 每次扫描行数
        :param batch_sleep_seconds: 每次扫描间歇（秒）
        :return:
        """
        start_post = 0
        logger.info("start to scan.")
        while True:
            stop_post, match_keys = self.scan_keys(
                key_match=key_match,
                start_post=start_post,
                batch_scan_size=batch_scan_size
            )
            logger.info("star post:{},stop post:{},scan keys:{},find keys:{}".format(
                start_post,
                stop_post,
                batch_scan_size,
                len(match_keys)
            ))
            if stop_post == 0:
                break
            start_post = stop_post
            time.sleep(batch_sleep_seconds)
        logger.info("stop to scan.")

    def clean_useless_key(self, key_match, use_less_seconds, min_expire_seconds,
                          max_random_seconds, batch_scan_size=1000, batch_sleep_seconds=0.1):
        """
        批量清理超过最大空闲时长的指定模式键值
        :param key_match: 键模式
        :param use_less_seconds: 最大空闲键值时长
        :param min_expire_seconds: 设置最小过期时长（秒）
        :param max_random_seconds: 设置最大随机时长(秒)
        :param batch_scan_size: 每次扫描行数
        :param batch_sleep_seconds: 每次扫描间歇（秒）
        :return:
        """
        start_post = 0
        logger.info("start to scan.")
        while True:
            stop_post, match_keys = self.scan_keys(
                key_match=key_match,
                start_post=start_post,
                batch_scan_size=batch_scan_size
            )
            logger.info("star post:{},stop post:{},scan keys:{},find keys:{}".format(
                start_post,
                stop_post,
                batch_scan_size,
                len(match_keys)
            ))
            for key_name in match_keys:
                idle_seconds = self.get_idle_seconds(key_name=key_name)
                if idle_seconds > use_less_seconds:
                    logger.info("键值{}当前的空闲时间为{},超过阈值{}".format(key_name, idle_seconds, use_less_seconds))
                    self.expire_key(
                        key_name=key_name,
                        min_expire_seconds=min_expire_seconds,
                        max_random_seconds=max_random_seconds
                    )
            if stop_post == 0:
                break
            start_post = stop_post
            time.sleep(batch_sleep_seconds)
        logger.info("stop to scan.")

    def clean_long_expired_key(
            self, key_match, min_expire_seconds, max_random_seconds,
            batch_scan_size=1000, batch_sleep_seconds=0.1):
        """
        对设置过期时间较长的KEY重新设置过期时间
        :param key_match: 键模式
        :param min_expire_seconds: 设置最小过期时长（秒）
        :param max_random_seconds: 设置最大随机时长(秒)
        :param batch_scan_size: 每次扫描行数
        :param batch_sleep_seconds: 每次扫描间歇（秒）
        :return:
        """
        start_post = 0
        logger.info("start to scan.")
        while True:
            stop_post, match_keys = self.scan_keys(
                key_match=key_match,
                start_post=start_post,
                batch_scan_size=batch_scan_size
            )
            logger.info("star post:{},stop post:{},scan keys:{},find keys:{}".format(
                start_post,
                stop_post,
                batch_scan_size,
                len(match_keys)
            ))
            for key_name in match_keys:
                key_ttl_seconds = self.get_ttl_seconds(key_name)
                if self.get_ttl_seconds(key_name) > min_expire_seconds:
                    logger.warning("key:{}的过期时间为{}，超过阈值{}".format(key_name, key_ttl_seconds, min_expire_seconds))
                    self.expire_key(
                        key_name=key_name,
                        min_expire_seconds=min_expire_seconds,
                        max_random_seconds=max_random_seconds
                    )
            if stop_post == 0:
                break
            start_post = stop_post
            time.sleep(batch_sleep_seconds)
        logger.info("stop to scan.")


def check_require_options(base_parser, base_options, require_options):
    for require_option in require_options:
        if (require_option not in base_options.__dict__.keys()) or (base_options.__dict__[require_option] is None):
            base_parser.error("--{} option require an argument".format(require_option))
            base_parser.print_help()
            exit()


def get_command_option():
    usage = """
#=======================================================#
用途: 通过设置过期时间将不需要的Key清理掉。
参数：
 --redis_host: 要清理的Redis服务地址, 默认值：127.0.0.1
 --redis_port: 要清理的Redis服务端口, 默认值: 6379
 --redis_auth: 要清理的Redis账号密码, 默认值：""
 --redis_db：  要清理的Redis DBA编号, 默认值：0
 --key_match:  要清理的Key名称匹配,   无默认值
 --exec_type:  要清理的操作类型，可选：
           clean_expired_key：清理已经过期Key,通过scan触发Redis自动清理。
           clean_idle_key: 清理长期空闲的Key,通过参数idle_seconds来指定空闲时间。
           clean_unexpired_key：清理未设置过期时间的Key。
           clean_long_expired_key：清理过期较长的Key, 对于过期时间超过参数min_expire_seconds指定的Key,将其过期时间设置为min_expire_seconds。
 --min_expire_seconds：设置要清理的Key的最小过期时间,单位为秒,无默认值。
 --max_random_seconds：设置要清理的key的最大随机时间,单位为秒,无默认值。由 min_expire_seconds + random(max_random_second)来确认最终过期时间。
 --batch_scan_size: 每次扫描的Key数量, 默认值为1000。
 --batch_sleep_seconds: 每次扫描操作的休眠间隔，单位为秒，默认值为0.1。
 --idle_seconds：设置判定Key是否空闲的时间,单位为秒,配合exec_type="clean_idle_key"来使用
#=======================================================#
"""
    # 使用使用add_help_option=True或使用默认值，都会自动添加help选项
    # parser = optparse.OptionParser(usage=usage)
    # 如果参数中使用到——h或主动添加--help参数，则使用add_help_option=False 关闭自动添加help选项
    parser = optparse.OptionParser(
        usage=usage,
        add_help_option=False,
        version="{} {}".format(APP_NAME, APP_VERSION)
    )
    parser.add_option(
        "--help",
        action="help",
        help="show help"
    )
    parser.add_option(
        "--redis_host", action="store", type="string", default='127.0.0.1',
        dest="redis_host", help="要清理的Redis服务地址,默认值：127.0.0.1"
    )
    parser.add_option(
        "--redis_port", type="int", dest="redis_port",
        default=6379, help="要清理的Redis服务端口,默认值：6379"
    )
    parser.add_option(
        "--redis_auth", action="store", type="string", dest="redis_auth",
        default="", help="要清理的Redis服务端口,默认值为空"
    )
    parser.add_option(
        "--redis_db", action="store", type="int", dest="redis_db",
        default=0, help="要清理的Redis DBA编号,默认值：0"
    )
    parser.add_option(
        "--exec_type", action="store", type="string", dest="exec_type",
        default="clean_expired_key", help="要清理的操作类型"
    )
    parser.add_option(
        "--min_expire_seconds", action="store", type="int", dest="min_expire_seconds",
        help="要清理的Redis DBA编号,必填项"
    )
    parser.add_option(
        "--max_random_seconds", action="store", type="int", dest="max_random_seconds",
        help="要清理的Redis DBA编号,必填项"
    )
    parser.add_option(
        "--idle_seconds", action="store", type="int", dest="idle_seconds",
        help="要清理的Redis DBA编号,必填项"
    )
    parser.add_option(
        "--batch_scan_size", action="store", type="int", dest="batch_scan_size",
        default=1000, help="每次扫描的Key数量, 默认值为1000"
    )
    parser.add_option(
        "--batch_sleep_seconds", action="store", type="float", dest="batch_sleep_seconds",
        default=0.1, help="每次扫描操作的休眠间隔，单位为秒，默认值为0.1。"
    )
    parser.add_option(
        "--key_match", action="store", type="string", dest="key_match",
        help="要清理的Key名称匹配,如要匹配前缀为um:cache的key,则设置为'um:cache*' "
    )
    (options, args) = parser.parse_args()
    check_require_options(
        base_parser=parser,
        base_options=options,
        require_options=["key_match", "exec_type"]
    )
    if options.exec_type == "clean_idle_key":
        check_require_options(
            base_parser=parser,
            base_options=options,
            require_options=["idle_seconds"]
        )
    if options.exec_type != "clean_all_key":
        check_require_options(
            base_parser=parser,
            base_options=options,
            require_options=["min_expire_seconds", "max_random_seconds"]
        )
    return options


def main():
    LoggerHelper.init_logger(logger_name=APP_NAME, logger_level=logging.INFO)
    global logger
    logger = LoggerHelper.get_logger(logger_name=APP_NAME)
    cmd_options = get_command_option()
    rh = RedisExpireHelper(
        redis_host=cmd_options.redis_host,
        redis_port=cmd_options.redis_port,
        redis_password=cmd_options.redis_auth,
        db_index=cmd_options.redis_db
    )
    if not rh.check_connection():
        logger.warning("Redis实例无法正常连接，请检查配置")
        exit(-1)
    if cmd_options.exec_type == "clean_idle_key":
        rh.clean_expired_key(
            key_match=cmd_options.key_match,
            batch_scan_size=cmd_options.batch_scan_size,
            batch_sleep_seconds=cmd_options.batch_sleep_seconds
        )

    if cmd_options.exec_type == "clean_expired_key":
        if cmd_options.idle_seconds < 60:
            logger.warning("参数clean_idle_key设置过小，无法执行")
            exit()
        rh.clean_useless_key(
            key_match=cmd_options.key_match,
            use_less_seconds=cmd_options.idle_seconds,
            min_expire_seconds=cmd_options.min_expire_seconds,
            max_random_seconds=cmd_options.max_random_seconds,
            batch_scan_size=cmd_options.batch_scan_size,
            batch_sleep_seconds=cmd_options.batch_sleep_seconds
        )

    if cmd_options.exec_type == "clean_unexpired_key":
        rh.clean_unexpired_key(
            key_match=cmd_options.key_match,
            min_expire_seconds=cmd_options.min_expire_seconds,
            max_random_seconds=cmd_options.max_random_seconds,
            batch_scan_size=cmd_options.batch_scan_size,
            batch_sleep_seconds=cmd_options.batch_sleep_seconds
        )

    if cmd_options.exec_type == "clean_long_expired_key":
        rh.clean_long_expired_key(
            key_match=cmd_options.key_match,
            min_expire_seconds=cmd_options.min_expire_seconds,
            max_random_seconds=cmd_options.max_random_seconds,
            batch_scan_size=cmd_options.batch_scan_size,
            batch_sleep_seconds=cmd_options.batch_sleep_seconds
        )


if __name__ == '__main__':
    main()
